

from selenium import webdriver
from selenium.webdriver import ChromeOptions
import pymongo

from spiders.beike_redis import logger

client = pymongo.MongoClient("localhost")
db = client['beike_house']
collection = db['wanghjing']
import time
import redis
# pool = redis.ConnectionPool(host='localhost0', port=6379, decode_responses=True)
# r = redis.Redis(connection_pool=pool)
import lxml
from lxml import etree

option = ChromeOptions()

# option.add_argument('--headless')
option.add_argument('--disable-infobars')  # 禁用浏览器正在被自动化程序控制的提示
# 反爬机制代码开始，采用此代码在F12控制台输入window.navigator.webdriver结果不是True，而是undefined就成功了
option.add_experimental_option('excludeSwitches', ['enable-automation'])
option.add_argument('--no-sandbox')
option.add_argument('--disable-dev-shm-usage')
option.add_argument('--headless')
option.add_argument('blink-settings=imagesEnabled=false')
# option.add_argument('--disable-gpu')
driver = webdriver.Chrome(options=option)
# self.driver = webdriver.PhantomJS() # 无界面浏览已停止更新，建议使用headless
# 反爬机制代码结束
# 窗口最大化
driver.maximize_window()
URL = "https://www.ke.com/city/"
driver.get(URL)
time.sleep(2)
driver.implicitly_wait(10)
# logger.info(self.driver.save_screenshot('image.png'))
# 往搜索框中输入iphone
data = driver.page_source
html = etree.HTML(data)
list_urls = []
list_names = []
for item in html.xpath("/html/body/div[3]/div[2]/div[1]/div[2]/ul/li"):
    logger.info(item)
    urls = item.xpath("div[2]/div/ul/li/a/@href")
    names = item.xpath("div[2]/div/ul/li/a/text()")
    urls = ["https:"+url for url in urls]
    list_names += names
    logger.info(names)
    logger.info(urls)
    list_urls += urls
    # urls = html.xpath("/html/body/div[3]/div[2]/div[1]/div[2]/ul/li[15]/div[2]/div/ul/li[22]/a")
# logger.info(list_urls)
zip_list = list(zip(list_names,list_urls))
logger.info(zip_list)
pool = redis.ConnectionPool(host='localhost', port=6379, db=7,decode_responses=True)
r = redis.Redis(connection_pool=pool)
for i in zip_list:
    logger.info(i)
    r.lpush("city_name_urls_queue",str({"city_name":i[0],"city_url":i[1]}))

